package com.kinth.work.zggys;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 
 * <p>
 * Title: 中国供应商网企业数据抓取
 * </p>
 * <p>
 * Description:
 * https://www3zzzcom.cn.china.cn/，这是具体的公司页面。（暂时没有发现其他类型的页面）。
 * 提取相关item.目前提取了
 * 公司名称，产品，地址，qq,联系方式，公司简介。微信是提取不出来的。因为它是直接跳出一个界面。
 *  由于程序太大。还没有系统测试过。单元测试完成。
 * </p>
 * 
 * @author kaikai
 * @2017年3月21日
 * 1. 网页有两种： 
 */

public class GetInfo implements PageProcessor, Runnable
{
	public static final int NUM_ONCE = 10;// 每次从数据库里面取出的值
	private static String companyUrlPath = System.getProperty("user.dir") + File.separator + "company.txt";
	private static File companyUrlFile = new File(companyUrlPath);
	private String industry;
	private String urlNow;
	private Site site = Site.me()
			.setUserAgent("Mozilla/5.0 (X11; U; Linux i686; zh-CN; rv:1.9.1.2) " + "Gecko/20090803").setRetryTimes(3)
			.setSleepTime(100);

	/**
	 * 1. 提取相关的item
	 * 
	 * @param page
	 */
	@Override
	public void process(Page page)
	{
		// 公司名称
		String name = page.getHtml()
				.xpath("//body/div[@class='content-wrapper']" + "/div[@class='bd']/div[@class='content fl-clr']"
						+ "/div[@class='con-lef']/div[@id='det_info']/div[@class='lef-bd']"
						+ "/ul/li[@class='comp']/h3/a/text()")
				.toString();
		if (name == null)
		{

			name = page
					.getHtml().xpath("//body" + "//div[@class='content fl-clr']"
							+ "//div[@id='det_info']/div[@class='lef-bd']" + "/ul//div[@class='comp']/a/text()")
					.toString();

		}
		String people = null;
		String address = null;
		String phone1 = null;
		String phone2 = null;
		String phone3 = null;
		String qq = null;
		String introduction = null;
		String product = null;

		if (name != null)
		{
			// 产品
			product = page.getHtml().xpath("//head/title/text()").toString();

			// 产品介绍
			introduction = page.getHtml().xpath("//body" + "//div[@class='content fl-clr']"
					+ "//div[@id='det_info']/div[@class='lef-bd']" + "/ul/li[5]").toString();
			if (introduction == null)
			{
				introduction = page.getHtml()
						.xpath("//body" + "//div[@class='content fl-clr']" + "//div[@class='company-info']/p/text()")
						.toString();
			}
			//替换取出提取的字符串里面的html代码
			String regex;
			regex = "<[\\s\\S]*?>";
			introduction = introduction.replaceAll(regex, "").trim();
			introduction = introduction.replaceAll("\n", "");

			// 对联系方式里面的利用循环处理
			int amount = 7;
			for (int i = 1; i <= amount; i++)
			{
				String tempKey = page
						.getHtml().xpath("//body" + "//div[@class='content fl-clr']"
								+ "//div[@id='contact']/div[@class='lef-bd']" + "/ul/li[" + i + "]/dl/dt/text()")
						.toString();
				String tempValue = page
						.getHtml().xpath("//body" + "//div[@class='content fl-clr']"
								+ "//div[@id='contact']/div[@class='lef-bd']" + "/ul/li[" + i + "]/dl/dd/text()")
						.toString();
				if (tempKey == null)
				{
					break;
				}
				if (tempKey.substring(0, 1).equals("联"))
				{
					people = tempValue;

				} else if (tempKey.substring(0, 1).equals("职"))
				{
					String people2 = tempValue;

					// 联系人加职位
					people = people + " " + people2;

				} else if (tempKey.substring(0, 1).equals("电"))
				{
					phone1 = tempKey + tempValue;

				} else if (tempKey.substring(0, 1).equals("手"))
				{
					phone2 = tempKey + tempValue;

				} else if (tempKey.substring(0, 1).equals("传"))
				{
					phone3 = tempKey + tempValue;

				} else if (tempKey.substring(0, 1).equals("地"))
				{
					address = tempKey + tempValue;

				}
				if (tempKey.substring(0, 1).equals("客"))
				{
					qq = page.getHtml()
							.xpath("//body" + "//div[@class='content fl-clr']"
									+ "//div[@id='contact']/div[@class='lef-bd']" + "/ul/li[" + i + "]/dl/dd/a")
							.toString();
					if (qq != null)
					{
						String qqTemp[] = qq.split("=");
						if (qqTemp.length >= 2)
						{
							qq = qqTemp[1];
							qqTemp = qq.split("\\(");
							if (qqTemp.length >= 2)
							{
								qq = qqTemp[1];
								qqTemp = qq.split(",");
								if (qqTemp.length >= 2)
								{
									qq = qqTemp[0];
								} else
								{
									qq = null;
								}
							} else
							{
								qq = null;
							}
						} else
						{
							qq = null;
						}
					}
				}

			}

		}

		page.putField("name", name);
		page.putField("product", product);
		page.putField("address", address);
		page.putField("people", people);
		page.putField("qq", qq);
		page.putField("introduction", introduction);
		page.putField("phone1", phone1);
		page.putField("phone2", phone2);
		page.putField("phone3", phone3);
		page.putField("industry", industry);
		page.putField("url", urlNow);
	}

	@Override
	public Site getSite()
	{
		return site;
	}

	// 一个代替main函数的函数
	public static void runSpider()
	{
		System.out.println("爬取具体信息开始");
		// 从记录文件里面取出记录
		int rank = FileUtil.getCompanyUrl2();
		// 取出，每次10个
		ArrayList<GongYingShang> urlList = dbUtil.queryUrl("gongyingshang_url3", rank, rank + NUM_ONCE);

		while (true)
		{
			System.out.println("新一次爬取具体信息开始");
			if (urlList.size() == 0)
			{
				try
				{
					Thread.sleep(1000 * 10);
					// 提醒数据库为空，以便完成爬取之后暂停
					System.out.println("GetInfo: 当下公司链接已经爬取完毕。请结合其他列表情况判断是否已经完成");
				} catch (InterruptedException e)
				{
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			} else
			{
				for (GongYingShang gongYingShang : urlList)
				{
					GetInfo getInfo = new GetInfo();
					String startUrlTemp = gongYingShang.getUrl();
					String industryTemp = gongYingShang.getIndustry();
					getInfo.industry = industryTemp;
					getInfo.urlNow = startUrlTemp;

					Spider.create(getInfo).addUrl(startUrlTemp).addPipeline(new DataPipeline()).run();
				}
			}

			int nextRank = FileUtil.getCompanyUrl2() + urlList.size();
			FileUtil.writeCompanyUrl2(nextRank);
			urlList = dbUtil.queryUrl("gongyingshang_url3", nextRank, nextRank + NUM_ONCE);
		}
	}

	public static void main(String[] args)
	{
		GetInfo getInfo = new GetInfo();

		// https://shengshunfaqi.cn.china.cn/
		// https://liu2001185.cn.china.cn/
		// https://www3zzzcom.cn.china.cn/
		//https://s500a3b.cn.china.cn/
		String startUrlTemp = "https://s500a3b.cn.china.cn/";
		String industryTemp = "测试";
		getInfo.industry = industryTemp;
		getInfo.urlNow = startUrlTemp;

		Spider.create(getInfo).addUrl(startUrlTemp)
				// .addPipeline(new DataPipeline())
				.run();
	}

	@Override
	public void run()
	{
		runSpider();

	}
}